﻿using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using Common.Utilities;
using Common.Utilities.Dispatcher;
using Common.Utilities.TextMatch;
using System.Net;
using System.IO;
using System.Xml;

namespace Workflows.Components.ETL
{
    /// <summary>
    /// 
    /// </summary>
    public class CitationParserWorker:IWorker 
    {
        #region Implementation of IWorker
        /// <summary></summary>
        public event WorkerStarted WorkerStartedEvent;
        /// <summary></summary>
        public event WorkerFinished WorkerFinishedEvent;
        /// <summary></summary>
        public event WorkerHasError WorkerErredEvent;
        /// <summary></summary>
        public event WorkerCancelled WorkerCancelledEvent;
        /// <summary></summary>
        public event WorkerStatusChanged WorkerStatusChangedEvent;

        private Thread workerThread;
        private CitationParserInstruction instruction;
        private CitationParserOutput output;
        private RunContext _Context;
        private Guid _TicketID;
        private string _WorkflowName;
        private string _ComponentName;

        /// <summary>
        /// 
        /// </summary>
        public string ComponentName
        {
            get { return _ComponentName; }
        }

        /// <summary>
        /// 
        /// </summary>
        public string WorkflowName
        {
            get { return _WorkflowName; }
        }

        /// <summary></summary>
        public Thread WorkerThread
        {
            get { return workerThread; }
        }
        /// <summary></summary>
        public InstructionBase Instruction
        {
            get { return instruction; }
        }
        /// <summary></summary>
        public ResultBase Output
        {
            get { return output; }
        }
        /// <summary></summary>
        public Type ComponentType
        {
            get { return typeof(CitationParserComponent); }
        }
        /// <summary></summary>
        public RunContext Context
        {
            get { return _Context; }
            set { _Context = value; }
        }
        /// <summary></summary>
        public void SetInstruction(Guid ticketGuid, InstructionBase request, string workflowName, string componentName)
        {
            this._TicketID = ticketGuid;
            this.instruction = (CitationParserInstruction)request;
            this._WorkflowName = workflowName;
            this._ComponentName = componentName;
        }
        /// <summary></summary>
        public void SetInstruction(RunContext context, Guid ticketGuid, InstructionBase request, string workflowName, string componentName)
        {
            this._TicketID = ticketGuid;
            this.instruction = (CitationParserInstruction)request;
            this._Context = context;
            this._WorkflowName = workflowName;
            this._ComponentName = componentName;
        }
        /// <summary></summary>
        public void DoWork()
        {
            if (this.WorkerStartedEvent != null)
            {
                this.WorkerStartedEvent(this._TicketID, this, this.GetType(), this.instruction);
            }
            this.workerThread = Thread.CurrentThread;
            this.ParseCitations();
        }
        /// <summary></summary>
        public void DoWorkAsync()
        {
            if (this.WorkerStartedEvent != null)
            {
                this.WorkerStartedEvent(this._TicketID, this, this.GetType(), this.instruction);
            }
            this.workerThread = new Thread(new ThreadStart(this.ParseCitations));
            this.workerThread.Start();
        }
        /// <summary></summary>
        public void CancelWork()
        {
            if (this.workerThread != null && this.workerThread.IsAlive)
            {
                this.workerThread.Abort();
                this.workerThread.Join();

                if (this.WorkerCancelledEvent != null)
                {
                    this.WorkerCancelledEvent(this._TicketID, this.GetType(), this.instruction);
                }
            }
        }

        #endregion

        #region work
        private void ParseCitations()
        {
            SqlConnection conn=new SqlConnection(this.instruction.ConnStr);
            SqlConnection conn2 = new SqlConnection(this.instruction.ConnStr);
            try
            {
                int totalRecords = DbUtil.GetTableRecordCount(
                DataProviderType.MSSQL, this.instruction.ConnStr, this.instruction.SQL);
                TableColumn keyCol = DbUtil.GetTableColumn(
                    DataProviderType.MSSQL, this.instruction.ConnStr, this.instruction.SQL, this.instruction.KeyField);
                string spName = string.Format("BulkInsert_{0}", this.instruction.OutputTable);
                List<TableColumn> cols = this.PrepareInsertCitationStoredProcedure(
                    this.instruction, spName, keyCol.DbType);

                conn.Open();
                int recordsProcessed = 0;
                SqlCommand cmd = conn.CreateCommand();
                cmd.CommandText = this.instruction.SQL;
                cmd.CommandType = CommandType.Text;

                conn2.Open();
                SqlCommand cmd2 = conn2.CreateCommand();
                cmd2.CommandText = spName;
                cmd2.CommandType = CommandType.StoredProcedure;
                CitationAPAStyles apaStyles = new CitationAPAStyles();
                CitationMLAStyles mlaStyles = new CitationMLAStyles();

                SqlDataReader reader = cmd.ExecuteReader();
                while (reader.Read())
                {
                    if (reader[this.instruction.CitationField] != DBNull.Value &&
                        reader[this.instruction.KeyField] != DBNull.Value)
                    {
                        try
                        {
                            object key = reader[this.instruction.KeyField];
                            string citationText = reader[this.instruction.CitationField].ToString();
                            Citation cite = new Citation();
                            if (this.instruction.Algorithm==CitationParserAlgorithms.Paracite &&
                                this.ExtractCitation(citationText, ref cite))
                            {
                                WosItem wosItem = this.LinkToWos(cite);
                                if(wosItem !=null)
                                    cite.WosItemID = wosItem.SourceItemID;
                                this.InsertCitation(cmd2, cite, this.instruction.SrcKeyField, key);
                                recordsProcessed++;
                            }
                            else if(this.ExtractCitationUsingDecisionTree(citationText, ref cite))
                            {
                                // default to using decision tree
                                WosItem wosItem = this.LinkToWos(cite);
                                if (wosItem != null)
                                    cite.WosItemID = wosItem.SourceItemID;
                                this.InsertCitation(cmd2, cite, this.instruction.SrcKeyField, key);
                                recordsProcessed++;
                            }

                            if (this.WorkerStatusChangedEvent != null && recordsProcessed % 100 == 0)
                            {
                                string msg = string.Format("Processed {0} of {1} records..", recordsProcessed,
                                                           totalRecords);
                                int pct = recordsProcessed * 100 / totalRecords;
                                this.WorkerStatusChangedEvent(this._TicketID, this.GetType(), this.instruction,
                                                              this._WorkflowName, this._ComponentName, msg, pct);
                            }
                        }
                        catch (Exception ex1)
                        {
                            if (this.WorkerErredEvent != null)
                                this.WorkerErredEvent(this._TicketID, this.GetType(), this.instruction, ex1.ToString(), false);
                        }
                    }
                }
                reader.Close();

                this.output = new CitationParserOutput(
                    this.instruction.OwnerComponentID, this._TicketID,
                    totalRecords, recordsProcessed);
                if (this.WorkerFinishedEvent != null)
                    this.WorkerFinishedEvent(this._TicketID, this.GetType(), this.instruction, this.output);
            }
            catch (Exception ex)
            {
                if (this.WorkerErredEvent != null)
                    this.WorkerErredEvent(this._TicketID, this.GetType(), this.instruction, ex.ToString(), true);
            }
            finally
            {
                conn.Close();
                conn2.Close();
            }
        }

        private List<TableColumn> PrepareInsertCitationStoredProcedure(
            CitationParserInstruction citeInstruction, 
            string spName,
            SqlDbType keyDbType)
        {
            List<TableColumn> cols=new List<TableColumn>();
            TableColumn col=new TableColumn();
            col.ColumnName = citeInstruction.SrcKeyField;
            col.DbType = keyDbType;
            col.Size = FieldDataType.GetFieldDefaultSize(col.DbType);
            cols.Add(col);
            // pub time
            col = new TableColumn();
            col.ColumnName = citeInstruction.PubTimeField;
            col.DbType = SqlDbType.VarChar;
            col.Size = FieldDataType.GetFieldDefaultSize(col.DbType);
            cols.Add(col);
            // title
            col = new TableColumn();
            col.ColumnName = citeInstruction.TitleField;
            col.DbType = SqlDbType.VarChar;
            col.Size = 1024;
            cols.Add(col);
            // book title meeting
            col = new TableColumn();
            col.ColumnName = citeInstruction.BookTitleMeetingField;
            col.DbType = SqlDbType.VarChar;
            col.Size = 1024;
            cols.Add(col);
            // Authors
            col = new TableColumn();
            col.ColumnName = citeInstruction.AuthorsField;
            col.DbType = SqlDbType.VarChar;
            col.Size = 1024;
            cols.Add(col);
            // Journal
            col = new TableColumn();
            col.ColumnName = citeInstruction.JournalNameField;
            col.DbType = SqlDbType.VarChar;
            col.Size = 1024;
            cols.Add(col);
            // Volume
            col = new TableColumn();
            col.ColumnName = citeInstruction.VolumeField;
            col.DbType = SqlDbType.VarChar;
            col.Size = FieldDataType.GetFieldDefaultSize(col.DbType);
            cols.Add(col);
            // Issue
            col = new TableColumn();
            col.ColumnName = citeInstruction.IssueField;
            col.DbType = SqlDbType.VarChar;
            col.Size = FieldDataType.GetFieldDefaultSize(col.DbType);
            cols.Add(col);
            // Pages
            col = new TableColumn();
            col.ColumnName = citeInstruction.PageField;
            col.DbType = SqlDbType.VarChar;
            col.Size = FieldDataType.GetFieldDefaultSize(col.DbType);
            cols.Add(col);
            // wos item
            col=new TableColumn();
            col.ColumnName = citeInstruction.WosItemIDField;
            col.DbType = SqlDbType.Int;
            col.Size = FieldDataType.GetFieldDefaultSize(col.DbType);
            cols.Add(col);

            string spBody = StoredProcUtil.CreateStoredProcedureStatementForInsert(
                DataProviderType.MSSQL, spName, "dbo", citeInstruction.OutputTable,
                cols.ToArray());
            bool created = DbUtil.NewStoredProc(citeInstruction.ConnStr, spName, spBody);
            if(!created)
                throw new Exception("Unable to create stored procedure for inserting output records");
            return cols;
        }

        private void InsertCitation(SqlCommand cmd, Citation citation, 
            string keyFieldName, object keyValue)
        {
            cmd.Parameters.Clear();
            cmd.Parameters.AddWithValue(keyFieldName, keyValue);
            if (citation.PubTime != null && citation.PubTime.Length > 255)
                citation.PubTime = citation.PubTime.Substring(0, 255);
            cmd.Parameters.AddWithValue(Citation.PubTimeField, citation.PubTime?? "");
            if (citation.Title != null && citation.Title.Length > 1024)
                citation.Title = citation.Title.Substring(0, 1024);
            cmd.Parameters.AddWithValue(Citation.TitleField, citation.Title ?? "");
            if (citation.BookTitleOrMeetingName != null && citation.BookTitleOrMeetingName.Length > 1024)
                citation.BookTitleOrMeetingName = citation.BookTitleOrMeetingName.Substring(0, 1024);
            cmd.Parameters.AddWithValue(Citation.BookTitleMeetingField, citation.BookTitleOrMeetingName ?? "");
            if (citation.Authors != null && citation.Authors.Length > 1024)
                citation.Authors = citation.Authors.Substring(0, 1024);
            cmd.Parameters.AddWithValue(Citation.AuthorField, citation.Authors ?? "");
            if (citation.Journal != null && citation.Journal.Length > 1024)
                citation.Journal = citation.Journal.Substring(0, 1024);
            cmd.Parameters.AddWithValue(Citation.JournalField, citation.Journal ?? "");
            if (citation.Volume != null && citation.Volume.Length > 50)
                citation.Volume = citation.Volume.Substring(0, 50);
            cmd.Parameters.AddWithValue(Citation.VolumeField, citation.Volume ?? "");
            if (citation.Issue != null && citation.Issue.Length > 50)
                citation.Issue = citation.Issue.Substring(0, 50);
            cmd.Parameters.AddWithValue(Citation.IssueField, citation.Issue ?? "");
            if (citation.Pages != null && citation.Pages.Length > 50)
                citation.Pages = citation.Pages.Substring(0, 50);
            cmd.Parameters.AddWithValue(Citation.PageField, citation.Pages ?? "");
            if (citation.WosItemID.HasValue)
                cmd.Parameters.AddWithValue(Citation.WosItemIDField, citation.WosItemID.Value);
            else
                cmd.Parameters.AddWithValue(Citation.WosItemIDField, DBNull.Value);
            cmd.ExecuteNonQuery();
        }
        #endregion

        #region test
        /// <summary>
        /// APA journal -> MLA journal -> APA book -> APA web -> MLA book -> MLA web
        /// </summary>
        /// <param name="citationText"></param>
        /// <param name="citation"></param>
        /// <param name="apaStyles"></param>
        /// <param name="mlaStyles"></param>
        /// <returns></returns>
        public bool ExtractCitation(
            string citationText, ref Citation citation,
            CitationAPAStyles apaStyles, CitationMLAStyles mlaStyles)
        {
            citation=new Citation();
            Dictionary<int, PatternWeight> patternWeights;
            if(this.MatchStyle(citationText, apaStyles.JournalArticle))
            {
                patternWeights = apaStyles.JournalArticle;
            }
            else if (this.MatchStyle(citationText, mlaStyles.JournalArticle))
            {
                patternWeights=mlaStyles.JournalArticle;
            }
            else if (this.MatchStyle(citationText, apaStyles.Book_Article))
            {
                patternWeights = apaStyles.Book_Article;
            }
            else if (this.MatchStyle(citationText, apaStyles.Website_Article))
            {
                patternWeights = apaStyles.Website_Article;
            }
            else if (this.MatchStyle(citationText,mlaStyles.Book_Article))
            {
                patternWeights = mlaStyles.Book_Article;
            }
            else if (this.MatchStyle(citationText, mlaStyles.Website_Article))
            {
                patternWeights = mlaStyles.Website_Article;
            }
            else
            {
                return false;
            }

            if(patternWeights !=null && patternWeights.Count>0)
            {
                citation = this.ExtractCitation(citationText, patternWeights);
            }
            return true;
        }

        private Citation ExtractCitation(string citationText, Dictionary<int,PatternWeight> patternWeights)
        {
            Citation citation=new Citation();
            List<int> ordersAPA = new List<int>();
            ordersAPA.AddRange(patternWeights.Keys);
            ordersAPA.Sort();
            foreach (int order in ordersAPA)
            {
                PatternWeight pattern = patternWeights[order];
                Regex regex = new Regex(pattern.Pattern, RegexOptions.IgnoreCase);
                List<string> phrases = SentenceBoundaryUtil.GetCitationPhrases(citationText);
                foreach (string phrase in phrases)
                {
                    Match match = regex.Match(phrase);
                    if (match.Success)
                    {
                        int fromPos = match.Index;
                        int toPos = match.Index + match.Value.Length;
                        if (pattern.MultiMatches)
                        {
                            while (match.Success)
                            {
                                toPos = match.Index + match.Value.Length;
                                match = match.NextMatch();
                            }
                        }
                        string matchedValue = phrase.Substring(fromPos, toPos - fromPos);
                        if (phrase.Length - matchedValue.Length > 3)
                            citationText = citationText.Replace(matchedValue, "");
                        else
                            citationText = citationText.Replace(phrase, "");
                        if (pattern.Key == Citation.AuthorField)
                            citation.Authors = matchedValue;
                        else if (pattern.Key == Citation.TitleField)
                            citation.Title = matchedValue;
                        else if (pattern.Key == Citation.JournalField)
                            citation.Journal = matchedValue;
                        else if (pattern.Key == Citation.VolumeField)
                            citation.Volume = matchedValue;
                        else if (pattern.Key == Citation.IssueField)
                            citation.Issue = matchedValue;
                        else if (pattern.Key == Citation.PageField)
                            citation.Pages = matchedValue;
                        else if (pattern.Key == Citation.PubTimeField)
                            citation.PubTime = matchedValue;
                        break;
                    }
                }
            }
            return citation;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="citationText"></param>
        /// <param name="apaStyles"></param>
        /// <param name="mlaStyles"></param>
        /// <returns></returns>
        public List<Citation> ExtractCitations(string citationText,
            CitationAPAStyles apaStyles, CitationMLAStyles mlaStyles)
        {
            string[] lines = citationText.Split(new char[] {'\n', '\r'});
            List<Citation> citations = new List<Citation>();
            foreach(string line in lines)
            {
                if(line.Trim().Length>0)
                {
                    Citation citation=new Citation();
                    bool canParse = this.ExtractCitation(line.Trim(), ref citation,
                        apaStyles, mlaStyles);
                    if(canParse)
                        citations.Add(citation);
                }
            }
            return citations;
        }
        #endregion

        #region extractions
        /// <summary>
        /// 
        /// </summary>
        /// <param name="citationText"></param>
        /// <param name="patternWeights"></param>
        /// <returns></returns>
        private bool MatchStyle(string citationText, Dictionary<int, PatternWeight> patternWeights)
        {
            double totalWeight = 0;
            double matchWeight = 0;
            List<int> orders=new List<int>();
            orders.AddRange(patternWeights.Keys);
            orders.Sort();
            foreach (int order in orders)
            {
                PatternWeight patternWeight = patternWeights[order];
                Regex regex = new Regex(patternWeight.Pattern, RegexOptions.IgnoreCase);
                List<string> phrases = SentenceBoundaryUtil.GetCitationPhrases(citationText);
                bool canMatchPattern = false;
                foreach (string phrase in phrases)
                {
                    if (regex.IsMatch(phrase))
                    {
                        Match match = regex.Match(phrase);
                        int fromPos = match.Index;
                        int toPos = match.Index + match.Value.Length;
                        if(patternWeight.MultiMatches)
                        {
                            while(match.Success)
                            {
                                toPos = match.Index + match.Value.Length;
                                match = match.NextMatch();
                            }
                        }
                        string matchedValue = phrase.Substring(fromPos, toPos - fromPos);
                        if (phrase.Length - matchedValue.Length > 3)
                            citationText = citationText.Replace(matchedValue, "");
                        else
                            citationText = citationText.Replace(phrase, "");
                        matchWeight += patternWeight.Weight;
                        canMatchPattern = true;
                        break;
                    }
                }
                if (patternWeight.Required && !canMatchPattern)
                    return false;

                totalWeight += patternWeight.Weight;
            }
            if (totalWeight > 0 && matchWeight > 0)
            {
                double score = matchWeight / totalWeight;
                if (score >= 0.5)
                    return true;
                else
                    return false;
            }
            else
                return false;
        }
        #endregion

        #region using paracite
        /// <summary>
        /// 
        /// </summary>
        /// <param name="citationText"></param>
        /// <param name="citation"></param>
        /// <returns></returns>
        public bool ExtractCitation(string citationText, ref Citation citation)
        {
            try
            {
                string url = "http://freecite.library.brown.edu/citations/create";
                HttpWebRequest httpReq =(HttpWebRequest) HttpWebRequest.Create(url);
                httpReq.Method = "POST";
                httpReq.Accept = "text/xml";
                string body = string.Format("citation={0}", citationText);
                StreamWriter writer = new StreamWriter(httpReq.GetRequestStream());
                writer.Write(body);
                writer.Close();
                HttpWebResponse httpResp =(HttpWebResponse) httpReq.GetResponse();
                StreamReader reader = new StreamReader(httpResp.GetResponseStream());
                string response = reader.ReadToEnd();
                reader.Close();
                XmlDocument xDoc = new XmlDocument();
                xDoc.LoadXml(response);
                XmlNode citationNode = xDoc.DocumentElement.SelectSingleNode("citation");
                if (citationNode == null)
                {
                    return false;
                }
                XmlNodeList authorNodes = citationNode.SelectNodes("authors/author");
                citation.Authors = string.Empty;
                if (authorNodes != null && authorNodes.Count > 0)
                {
                    Regex yearRegex=new Regex("(19|20)\\d\\d");
                    foreach (XmlNode authorNode in authorNodes)
                    {
                        if (citation.Authors.Length > 0)
                            citation.Authors += ",";
                        if (yearRegex.IsMatch(authorNode.InnerText))
                        {
                            citation.Authors += yearRegex.Replace(authorNode.InnerText, "");
                            citation.PubTime = yearRegex.Match(authorNode.InnerText).Value;
                        }
                        else
                            citation.Authors += authorNode.InnerText;
                    }
                }
                citation.Title = string.Empty;
                XmlNode titleNode = citationNode.SelectSingleNode("title");
                if (titleNode != null)
                    citation.Title = titleNode.InnerText;
                citation.Journal = string.Empty;
                XmlNode journalNode = citationNode.SelectSingleNode("journal");
                if (journalNode != null)
                    citation.Journal = journalNode.InnerText;
                citation.Pages = string.Empty;
                XmlNode pageNode = citationNode.SelectSingleNode("pages");
                if (pageNode != null)
                    citation.Pages = pageNode.InnerText;
                XmlNode yearNode = citationNode.SelectSingleNode("year");
                if (yearNode != null)
                    citation.PubTime = yearNode.InnerText;
                citation.Volume = string.Empty;
                XmlNode volumeNode = citationNode.SelectSingleNode("volume");
                if (volumeNode != null)
                    citation.Volume = volumeNode.InnerText;
                citation.Issue = string.Empty;
                XmlNode numNode = citationNode.SelectSingleNode("number");
                if (numNode != null)
                    citation.Issue = numNode.InnerText;
                citation.BookTitleOrMeetingName = string.Empty;
                XmlNode bookTitleNode = citationNode.SelectSingleNode("booktitle");
                if (bookTitleNode != null)
                    citation.BookTitleOrMeetingName = bookTitleNode.InnerText;

                httpResp.Close();
                return true;
            }
            catch (Exception ex)
            {
                if (this.WorkerErredEvent != null)
                    this.WorkerErredEvent(this._TicketID, this.GetType(), this.instruction, ex.ToString(), false);
                return false;
            }
        }
        #endregion

        #region using decision tree
        /// <summary>
        /// 
        /// </summary>
        /// <param name="citationText"></param>
        /// <param name="citation"></param>
        /// <returns></returns>
        public bool ExtractCitationUsingDecisionTree(string citationText, ref Citation citation)
        {
            try
            {
                CitationStyleEncoding encoder=new CitationStyleEncoding();
                citation = encoder.Parse(citationText);
                if (!string.IsNullOrEmpty(citation.Journal))
                {
                    return true;
                }
                else
                    return false;
            }
            catch(Exception ex)
            {
                if (this.WorkerErredEvent != null)
                    this.WorkerErredEvent(this._TicketID, this.GetType(), this.instruction, ex.ToString(), false);
                return false;
            }
        }
        #endregion

        #region link to wos
        private WosItem LinkToWos(Citation cite)
        {
            try
            {
                if(string.IsNullOrEmpty(cite.Authors) || string.IsNullOrEmpty(cite.Journal))
                    return null;

                List<string> authorNames = WosLinker.SplitAuthors(cite.Authors);
                int pubYear;
                bool canParseYear = int.TryParse(cite.PubTime, out pubYear);
                if (!canParseYear)
                {
                    DateTime pubTime;
                    bool hasPubTime = DateTime.TryParse(cite.PubTime, out pubTime);
                    if (hasPubTime)
                        pubYear = pubTime.Year;
                }
                Range<int> pages = WosLinker.SplitPages(cite.Pages);
                
                WosItem wosItem =
                    WosLinker.FindLinkedWosItem(
                        authorNames, cite.Journal,
                        cite.Volume, cite.Issue, cite.Title,
                        pubYear, pages);
                return wosItem;
            }
            catch (Exception ex)
            {
                if (this.WorkerErredEvent != null)
                    this.WorkerErredEvent(this._TicketID, this.GetType(), this.instruction, ex.ToString(), false);
                return null;
            }
        }

        #endregion
    }
}
